import queue, threading
import time, random
from blog_spider import crawl, parse, urls

def do_crawl(url_queue: queue.Queue, html_queue: queue.Queue):
    while True:
        url = url_queue.get()
        html = crawl(url)
        html_queue.put(html)
        print(threading.current_thread().name, f'crawl {url}, url_queue.size: {url_queue.qsize()}')
        time.sleep(random.randint(1, 2))

def do_parse(html_queue: queue.Queue, fw):
    while True:
        url = html_queue.get()
        results = parse(url)
        for result in results:
            fw.write(str(result) + '\n')
        print(threading.current_thread().name, f'html_queue.size: {html_queue.qsize()}')
        time.sleep(random.randint(1, 2))

if __name__ == '__main__':
    url_queue = queue.Queue()
    html_queue = queue.Queue()
    for url in urls:
        url_queue.put(url)

    fw = open('01-result.txt', 'w')
    for i in range(3):
        t = threading.Thread(target=do_crawl, args=(url_queue, html_queue,), name=f'crawl_{i}')
        t.start()

    for i in range(2):
        t = threading.Thread(target=do_parse, args=(html_queue, fw,), name=f'parse_{i}')
        t.start()
